package com.terren.spider.core.html.pipeline;

import com.terren.spider.core.html.configuration.Configuration;
import com.terren.spider.core.html.scheme.ICustomScheme;
import com.terren.spider.entity.common.SpiderLog;
import com.terren.spider.entity.config.SysConfig;
import com.terren.spider.util.common.BeanUtil;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;


/** 新闻页面的pipeline 继承 ResultItemsCollectorPipeline
 * 
 * @author chengyu
 *
 * @param <T>
 */
public class HtmlDBCollectorPipeline<T> extends BasicHtmlCollectorPipeline<T> {
	
	
	
	public HtmlDBCollectorPipeline(Configuration configuration, Class<T> clazz) throws Exception{
		super(configuration, clazz);
	}

	@Override
    public synchronized void process(ResultItems resultItems, Task task) {
		
		if (resultItems.get(SysConfig.COLLECT_ITEM_KEY) instanceof Page) {
			Page page = resultItems.get(SysConfig.COLLECT_ITEM_KEY);
			Integer totalPage = resultItems.get(SysConfig.COLLECT_TOTAL_TARGET_URL);
			try {
				/**
				 * 分析前处理
				 */
				for (ICustomScheme	scheme : configuration.getSchemes()) {
					if (BeanUtil.includeMethod("preParse", scheme.getClass())) {
						scheme.preParse(page, configuration);
					}
				}
				this.collectItems.addAll(parseUtil.parseList(page, configuration.getEntry(),clazz));
				saveToDB(collectItems,totalPage);
			} catch (Exception e) {
				try {
					SpiderLog spiderLog = new SpiderLog();
					spiderLog.setMessage("[entry:"+configuration.getEntry().getEntryId()+"]save to db error:"+e.getMessage());
					spiderLog.setStatus(0);
					spiderLogService.addSpiderLog(spiderLog);
				} catch (Exception e1) {
					log.error("[entry:"+configuration.getEntry().getEntryId()+"]save to spider log error:"+e.getMessage(),e1);
				}
			}
		}
    }
}
